/* _____________________________________________________________________________

	Project: The Effects of Independent Local Radio on Tanzanian Public Opinion: Evidence from a Planned Natural Experiment
	Authors: Donald P. Green, Dylan Groves, Constantine Manda, Beatrice Montano, Bardia Rahmani
	
	Purpose: Analysis
	Date: 2022.10
________________________________________________________________________________*/


	
/* Introduction ________________________________________________________________*/
	
	clear all	
	set more off
	
/* Set Control Directory _______________________________________________________*/

	*global user "X:/Dropbox/Wellspring Tanzania Papers/Wellspring Tanzania - Natural Experiment/pfm_ne_replication"
			
/* Load Indices ________________________________________________________________*/ 

	do "${user}/pfm_ne_replication_indices.do"									// Note that you need to go here to look at set of outcomes stored in each index

/* Load Data ___________________________________________________________________*/	
	
	use "${user}/pfm_ne_replication_data.dta", replace 
	sort id_resp_uid
	set seed 1956
	
/* Define Globals and Locals ___________________________________________________*/

	#d ;
	
		/* Set seed */
		set seed 			1956
							;
								
		/* Indices */			
		local index_list	
							t1_uptake
							tA4_interestknowledge
							tA5_gender
							tA6_values
							tA7_interestknowledge
							tA8_gender
							tA9_values
							tA10_ptixpart
							tA11_nonpfm
							;
				
		/* Covariates */	
		global cov_always	
							i.block_ne
							svy_replacement
							;	
						
		/* Lasso Covariates */
		global cov_lasso	
							resp_age resp_female resp_muslim b_resp_standard7 b_resp_married ///
							b_resp_religiosity b_resp_literate b_resp_lang_swahili ///
							b_resp_numhh b_resp_numkid b_resp_numolder b_resp_numyounger b_resp_numadult ///
							b_ge_index b_ge_raisekids b_ge_earning b_ge_school b_ge_leadership b_ge_noprefboy ///
							b_em_reject_story b_fm_reject ///
							b_ipv_rej_index b_ipv_norm_rej b_ipv_wifereact ///
							b_radio_any b_radio_community_dum b_radio_news_dum ///
							b_values_likechange b_values_money b_values_techgood ///
							b_values_elders b_values_respectauthority b_values_individualism ///
							b_radio_programs_music b_radio_programs_sport b_radio_programs_news b_radio_programs_romance b_radio_programs_social b_radio_programs_relig ///
							b_s3q2_freetime_rest b_s3q2_freetime_bar b_s3q2_freetime_sportwatch b_s3q2_freetime_sportplay ///
							b_s3q2_freetime_tv b_s3q2_freetime_cell b_s3q2_freetime_radio b_s3q2_freetime_community ///
							b_s3q3_converse_elec b_s3q3_converse_ipv b_s3q3_converse_discr b_s3q3_converse_vote b_s3q3_converse_water b_s3q3_converse_teachers ///
							b_asset_cell b_asset_tv b_asset_chair b_asset_sofa b_asset_table b_asset_motorcycle b_asset_radio_num ///
							b_freetime_media b_discuss_ptix b_discuss_nothing b_radio_news b_dk_ptix ///
							b_collact b_contactgov ///
							v_cell v_cell_bar v_timetotown ///
							v_electricity v_muslim v_mosques ///
							v_churches v_totworship v_villexec v_pop v_subvills ///
							v_poplist v_poplist_final ///
							enum1 enum2 enum3 enum4 enum5 enum6 enum7 ///
							enum8 enum9 enum10 enum11 enum12 enum13 enum14 enum15 enum16 enum17 enum18 enum19 enum20 
							;
							
		global cov_lasso_replacement 							
							resp_muslim resp_female resp_age ///
							v_cell v_cell_bar v_timetotown ///
							v_electricity v_muslim v_mosques ///
							v_churches v_totworship v_villexec v_pop v_subvills ///
							v_poplist v_poplist_final  ///
							enum1 enum2 enum3 enum4 enum5 enum6 enum7 enum8 ///
							enum9 enum10 enum11 enum12 enum13 enum14 enum15 ///
							enum16 enum17 enum18 enum19 enum20
							;
	#d cr

	
/* Create Interaction Variables ________________________________________________*/

	foreach var of global cov_lasso {
		qui gen `var'_int = `var' * svy_nonreplacement
	}

	foreach var of global cov_lasso_replacement {
		qui gen `var'_int_replace = `var' * svy_replacement
	}

/* Standardize lasso covariates ________________________________________________*/
	
	foreach var of global cov_lasso {
		qui egen std_`var' = std(`var')
		qui replace `var' = std_`var'
		qui drop std_`var'
	}

	foreach var of global cov_lasso_replacement {
		qui egen std_`var' = std(`var')
		qui replace `var' = std_`var'
		qui drop std_`var'
	}

	
/* Run for Each Index __________________________________________________________*/

global table_num 1

foreach index of local index_list {

	sort id_resp_uid
	set seed 1956

	/* Drop Macros */
	macro drop lasso_ctls 
	macro drop lasso_ctls_num 
	macro drop lasso_ctls_int
	
	macro drop lasso_ctls_replacement
	macro drop lasso_ctls_num_replacement 
	macro drop lasso_ctls_int_replace

	
	/* Define Matrix _______________________________________________________________*/
				
		/* Set Put Excel File Name */
		putexcel clear
		putexcel set "${user}/pfm_ne_replication_rawoutput_main.xlsx", sheet(`index', replace) modify
		
		qui putexcel A1 = ("variable")
		qui putexcel B1 = ("variablelabel")
		qui putexcel C1 = ("coef")
		qui putexcel D1 = ("se")
		qui putexcel E1 = ("pval")
		qui putexcel F1 = ("r2")
		qui putexcel G1 = ("N")
		qui putexcel H1 = ("lasso_coef")
		qui putexcel I1 = ("lasso_se")
		qui putexcel J1 = ("lasso_pval")
		qui putexcel K1 = ("lasso_r2")
		qui putexcel L1 = ("lasso_N")
		qui putexcel M1 = ("lasso_ctls")
		qui putexcel N1 = ("lasso_ctls_num")
		qui putexcel O1 = ("treat_mean")
		qui putexcel P1 = ("treat_sd")
		qui putexcel Q1 = ("ctl_mean")
		qui putexcel R1 = ("ctl_sd")
		qui putexcel S1 = ("vill_sd")
		qui putexcel T1 = ("min")
		qui putexcel U1 = ("max")
		qui putexcel V1 = ("lasso_ctls_replacement")
		qui putexcel W1 = ("lasso_ctls_num_replacement")
		qui putexcel X1 = ("boot_p")
		qui putexcel Y1 = ("l_boot_p")
		qui putexcel Z1 = ("test")
		qui putexcel AA1 = ("lowb")
		qui putexcel AB1 = ("uppb")
		qui putexcel AC1 = ("l_lowb")
		qui putexcel AD1 = ("l_uppb")
		qui putexcel AE1 = ("corr")


		/* Take list of DVs in the index _______________________________________*/
		
		local var_list ${`index'}		// Variables
		local row = 2					// Row for exporting to matrix
		local colum = 1
		
		/* Run regressions for each variable */
		
		foreach dv of local var_list  {
		
		/* set test */															// Note that these tests are established in the index do file
		if strpos("${twosided}", "`dv'") { 
			global test twosided 
		} 
			else {
				global test onesided
			}

		/* Variable name */
		di "*** VARIABLE IS `dv' ****	"
		ds `dv'
			global varname = "`r(varlist)'"  
			
		/* Variable label */
		global varlabel : var label `dv'
		
		/* Control mean */
		di "Control mean"
		sum `dv' if treat == 0 
			global ctl_mean `r(mean)'
			global ctl_sd `r(sd)'

		/* Treatment mean */
		di "Treatment mean"
		sum `dv' if treat == 1 
			global treat_mean `r(mean)'
			global treat_sd `r(sd)'
		
		/* Control village sd */
		di "Control Village SD"
		preserve
		qui collapse (mean) `dv' treat, by(id_village_uid)
		sum `dv' if treat == 0
			global vill_sd : di %6.3f r(sd)
		restore

		/* Variable range */	
		di "Variable Range"
		sum `dv' 
			global min = r(min)
			global max = r(max)
	
	/* Run standard regression _________________________________________________*/
	
		sort id_resp_uid
		reg `dv' treat ${cov_always}, cluster(id_village_uid)
			matrix table = r(table)
			
		if "$test" == "onesided" {
			sort id_resp_uid
			boottest treat, p(upper) nograph seed(1956)
				global boot_p = r(p)
		}
		
		if "$test" == "twosided" {
			sort id_resp_uid
			boottest treat, p(equaltail) nograph seed(1956)
				global boot_p = r(p)
		}
				
			/* Save values from regression */
			global coef 	= table[1,1]    	//beta
			global se 		= table[2,1]		//se
			global t 		= table[3,1]		//t
			global pval 	= table[4,1]		//twosided pval
			global lowb		= table[5,1]		//95 conf int lower bound
			global uppb		= table[6,1]		//95 conf int upper bound
			global r2 		= `e(r2_a)' 		//r-squared
			global n 		= e(N) 				//N
			
	/* Run lasso regression ____________________________________________________*/

		/* Run boot tests lasso  
		qui lasso linear `dv' ${cov_lasso}
			global lasso_ctls = e(allvars_sel)
			global lasso_ctls_num = e(k_nonzero_sel)
			
		if ${lasso_ctls_num} != 0 {
			reg `dv' treat 	${lasso_ctls} ${cov_always}, cluster(id_village_uid)
					matrix table = r(table)
					
					qui boottest treat, p(upper)
						global l_boot_p = r(p)
		}
		
		if ${lasso_ctls_num} == 0 {
			reg `dv' treat ${cov_always}, cluster(id_village_uid)
					matrix table = r(table)
					
					qui boottest treat, p(upper)
						global l_boot_p = r(p)
		}
		*/
		
		/* Run and save lasso for non-replacements */
		sort id_resp_uid
		set seed 1956
		qui lasso linear `dv' ${cov_lasso} if svy_replacement == 0, rseed(1956)
			global lasso_ctls = e(allvars_sel)
			global lasso_ctls_num = e(k_nonzero_sel)
			
		/* Run and save lasso for replacements */
		sort id_resp_uid
		set seed 1956
		qui lasso linear `dv' ${cov_lasso_replacement} if svy_replacement == 1, rseed(1956)
			global lasso_ctls_replacement = e(allvars_sel)
			global lasso_ctls_num_replacement 	= e(k_nonzero_sel)
		
		/* Create interaction term globals */
		if ${lasso_ctls_num} != 0 {							
			foreach ctl_var of global lasso_ctls {
				global lasso_ctls_int ${lasso_ctls_int} `ctl_var'_int
			}
		}
		
		/* Create interaction terms globals for replacement */
		if ${lasso_ctls_num_replacement} != 0 {
			foreach ctl_var of global lasso_ctls_replacement {
				global lasso_ctls_int ${lasso_ctls_int_replace} `ctl_var'_int_replace
			}
		}
			

	
di "****************************************************************************"
di "****************************************************************************"
di "****************************************************************************"			
di "****** TABLE ${table_num}: `index', COLUMN `colum': `dv' *******************"
di "****************************************************************************"
di "****************************************************************************"
di "****************************************************************************"
		
		
				/* If lasso selected covariates for both */
				if ${lasso_ctls_num} != 0 & ${lasso_ctls_num_replacement} != 0 {
					reg `dv' treat 	${lasso_ctls} ${lasso_ctls_int} ///
										${svy_ctls_replace} ${lasso_ctls_int_replace} svy_replacement ///
										${cov_always}, cluster(id_village_uid)
					matrix table = r(table)
					
					sort id_resp_uid
					boottest treat, p(upper) nograph seed(1956)
						global l_boot_p = r(p)
						
					di "LASSO SELECTED COVARIATES FOR BOTH"
				}
				
				/* If lasso selected covariates for only non-replacements */
				if ${lasso_ctls_num} != 0 & ${lasso_ctls_num_replacement} == 0 {		// If no lassovars chosen
					reg `dv' treat 	${lasso_ctls} ${lasso_ctls_int} ///
										svy_replacement ///
										${cov_always}, cluster(id_village_uid)
					matrix table = r(table)
					
					sort id_resp_uid
					boottest treat, p(upper) nograph seed(1956)
						global l_boot_p = r(p)	
						
					di "LASSO SELECTED COVARIATES NON-REPLACEMENTS"
				}
				
				/* If lasso selected covariates for only replacements */
				if ${lasso_ctls_num} == 0 & ${lasso_ctls_num_replacement} != 0 {		// If no lassovars chosen
					reg `dv' treat 	${svy_ctls_replace} ${lasso_ctls_int_replace} svy_replacement ///
										${cov_always}, cluster(id_village_uid)
					matrix table = r(table)
					
					sort id_resp_uid
					boottest treat , nograph seed(1956)
						global l_boot_p = r(p)
						
					di "LASSO SELECTED COVARIATES ONLY REPLACEMENTS"
				}
				
				/* If lasso selected no covariates */
				if ${lasso_ctls_num} == 0 & ${lasso_ctls_num_replacement} == 0 {		// If no lassovars chosen
					reg `dv' treat 	svy_replacement ///
										${cov_always}, cluster(id_village_uid)
					matrix table = r(table)
					
					sort id_resp_uid
					boottest treat , nograph seed(1956)
						global l_boot_p = r(p)
						
					di "NO LASSO SELECTED COVARIATES"
				}
				
			/* Save values from regression */
			global lasso_coef 	= table[1,1]    	//beta
			global lasso_se 	= table[2,1]		//se
			global lasso_t 		= table[3,1]		//t
			global l_lowb		= table[5,1]		//95 conf int lower bound
			global l_uppb		= table[6,1]		//95 conf int upper bound
			global lasso_r2 	= `e(r2_a)' 		//r-squared
			global lasso_n 		= e(N) 				//N
			
			/* One-sided p-value for predicted effects */
			if table[1,1] > 0 {
				global lasso_pval = ttail(e(df_r),abs(${lasso_t}))
				global help "One-tailed"
			}
			else if table[1,1] < 0 {
				global lasso_pval = 1-ttail(e(df_r),abs(${lasso_t}))
				global help "Two-tailed"
			}
			
			
	/* Export to Excel _________________________________________________________*/ 
		
		di "Variable is ${varname}, coefficient is ${coef}, pval is ${pval}, bootstrappedpval is ${boot_p}, N = ${n}"
		di "LASSO: Variable is ${varname}, coefficient is ${lasso_coef}, pval is ${lasso_pval}, N = ${lasso_n}"
		di "LASSO vars were ${lasso_ctls}"

		qui putexcel A`row' = ("${varname}")
		qui putexcel B`row' = ("${varlabel}")
		qui putexcel C`row' = ("${coef}")
		qui putexcel D`row' = ("${se}")
		qui putexcel E`row' = ("${pval}")
		qui putexcel F`row' = ("${r2}")
		qui putexcel G`row' = ("${n}")
		qui putexcel H`row' = ("${lasso_coef}")
		qui putexcel I`row' = ("${lasso_se}")
		qui putexcel J`row' = ("${lasso_pval}")
		qui putexcel K`row' = ("${lasso_r2}")
		qui putexcel L`row' = ("${lasso_n}")
		qui putexcel M`row' = ("${lasso_ctls}")
		qui putexcel N`row' = ("${lasso_ctls_num}")
		qui putexcel O`row' = ("${treat_mean}")
		qui putexcel P`row' = ("${treat_sd}")
		qui putexcel Q`row' = ("${ctl_mean}")
		qui putexcel R`row' = ("${ctl_sd}")
		qui putexcel S`row' = ("${vill_sd}")
		qui putexcel T`row' = ("${min}")
		qui putexcel U`row' = ("${max}")
		qui putexcel V`row' = ("${lasso_ctls_replacement}")
		qui putexcel W`row' = ("${lasso_ctls_num_replacement}")
		qui putexcel X`row' = ("${boot_p}")
		qui putexcel Y`row' = ("${l_boot_p}")
		qui putexcel Z`row' = ("${test}")
		qui putexcel AA`row' = ("${lowb}")
		qui putexcel AB`row' = ("${uppb}")
		qui putexcel AC`row' = ("${l_lowb}")
		qui putexcel AD`row' = ("${l_uppb}")
		qui putexcel AE`row' = ("${corr_naive}")
		
	/* Update locals ___________________________________________________________*/
		
		local row = `row' + 1
		local colum = `colum' + 1
	}
	
		global table_num = ${table_num} + 1

}













